<?php declare(strict_types=1);
/**
 * @author      xianganyall <xianganyall@gmail.com>
 * @copyright   2023-2025 owner
 **/

namespace Srv\Libs\Plugins\Gpt;

use Srv\Libs\Common\CommJson;

final class Gpt3Encoder
{
    /**
     * @param string $text
     * @return array
     * gptEncodeList
     */
    public static function gptEncodeList(string $text):array
    {
        $bpeTokens  = [];
        if(strlen($text) < 1) return $bpeTokens;
        $rawChars   = @file_get_contents(dirname(__FILE__).'/characters.json');
        $byteEncoder= is_string($rawChars) ? CommJson::decodeArray($rawChars) : [];
        if(count($byteEncoder) < 1) return $bpeTokens;
        $renCoder   = @file_get_contents(dirname(__FILE__).'/encoder.json');
        $encoder    = is_string($renCoder) ? CommJson::decodeArray($renCoder) : [];
        if(count($encoder) < 1) return $bpeTokens;
        $bpeFile    = @file_get_contents(dirname(__FILE__).'/vocab.bpe');
        if(!is_string($bpeFile) || strlen($bpeFile) < 1) return $bpeTokens;
        preg_match_all("#'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+#u", $text, $matches);
        if(!isset($matches[0]) || count($matches[0]) < 1) return $bpeTokens;
        $lines          = preg_split('/\r\n|\r|\n/', $bpeFile);
        $bpeMerges      = [];
        $bpeMergesTemp  = array_slice($lines, 1, count($lines), true);
        foreach($bpeMergesTemp as $bmt){
            $splitBmt   = preg_split('#(\s+)#', $bmt);
            $splitBmt   = array_filter($splitBmt, [__CLASS__, 'gptMyFilter']);
            if(count($splitBmt) > 0) $bpeMerges[] = $splitBmt;
        }
        $bpeRanks   = self::gptDictZip($bpeMerges, range(0, count($bpeMerges) - 1));
        $cache      = [];
        foreach($matches[0] as $token){
            $newTokens  = [];
            $chars      = [];
            $token      = mb_convert_encoding($token, 'UTF-8', 'ISO-8859-1');
            if(function_exists('mb_strlen')){
                $len            = mb_strlen($token, 'UTF-8');
                for ($i = 0; $i < $len; $i++){
                    $chars[]    = mb_substr($token, $i, 1, 'UTF-8');
                }
            }else{
                $chars          = str_split($token);
            }
            $result_word        = '';
            foreach($chars as $char){
                if(isset($byteEncoder[self::gptUnichr($char)])) $result_word .= $byteEncoder[self::gptUnichr($char)];
            }
            $newTokensBpe = self::gptBpe($result_word, $bpeRanks, $cache);
            foreach(explode(' ', $newTokensBpe) as $x){
                if(isset($encoder[$x])){
                    $newTokens[$x] = $encoder[$x];
                }else{
                    $newTokens[$x] = $x;
                }
            }
            foreach($newTokens as $ninx => $nval){
                if(isset($bpeTokens[$ninx])){
                    $bpeTokens[] = $nval;
                }else{
                    $bpeTokens[$ninx] = $nval;
                }
            }
        }
        return $bpeTokens;
    }

    /**
     * @param $var
     * @return bool
     * gptMyFilter
     */
    private static function gptMyFilter($var):bool
    {
        return ($var !== NULL && $var !== FALSE && $var !== '');
    }

    /**
     * @param $c
     * @return float|int
     * gptUnichr
     */
    private static function gptUnichr($c)
    {
        if (ord($c[0]) >=0 && ord($c[0]) <= 127) return ord($c[0]);
        if (ord($c[0]) >= 192 && ord($c[0]) <= 223) return (ord($c[0])-192)*64 + (ord($c[1])-128);
        if (ord($c[0]) >= 224 && ord($c[0]) <= 239) return (ord($c[0])-224)*4096 + (ord($c[1])-128)*64 + (ord($c[2])-128);
        if (ord($c[0]) >= 240 && ord($c[0]) <= 247) return (ord($c[0])-240)*262144 + (ord($c[1])-128)*4096 + (ord($c[2])-128)*64 + (ord($c[3])-128);
        if (ord($c[0]) >= 248 && ord($c[0]) <= 251) return (ord($c[0])-248)*16777216 + (ord($c[1])-128)*262144 + (ord($c[2])-128)*4096 + (ord($c[3])-128)*64 + (ord($c[4])-128);
        if (ord($c[0]) >= 252 && ord($c[0]) <= 253) return (ord($c[0])-252)*1073741824 + (ord($c[1])-128)*16777216 + (ord($c[2])-128)*262144 + (ord($c[3])-128)*4096 + (ord($c[4])-128)*64 + (ord($c[5])-128);
        if (ord($c[0]) >= 254 && ord($c[0]) <= 255) return 0;
        return 0;
    }

    /**
     * @param $x
     * @param $y
     * @return array
     * gptDictZip
     */
    private static function gptDictZip($x, $y):array
    {
        unset($y);
        $result = [];
        $cnt = 0;
        foreach($x as $i){
            if(isset($i[1]) && isset($i[0])){
                $result[$i[0] . ',' . $i[1]] = $cnt;
                $cnt++;
            }
        }
        return $result;
    }

    /**
     * @param $word
     * @return array
     * gptGetPairs
     */
    private static function gptGetPairs($word):array
    {
        $pairs = [];
        $prev_char = $word[0];
        for ($i = 1; $i < count($word); $i++)
        {
            $char = $word[$i];
            $pairs[] = array($prev_char, $char);
            $prev_char = $char;
        }
        return $pairs;
    }

    /**
     * @param $str
     * @param int $len
     * @return array
     * gptSplit
     */
    private static function gptSplit($str, int $len = 1):array
    {
        $arr		    = [];
        if(function_exists('mb_strlen')){
            $length 	= mb_strlen($str, 'UTF-8');
        }else{
            $length 	= strlen($str);
        }
        for ($i = 0; $i < $length; $i += $len){
            if(function_exists('mb_substr')){
                $arr[] = mb_substr($str, $i, $len, 'UTF-8');
            }else{
                $arr[] = substr($str, $i, $len);
            }
        }
        return $arr;
    }

    /**
     * @param $token
     * @param $bpeRanks
     * @param $cache
     * @return mixed|string
     * gptBpe
     */
    private static function gptBpe($token, $bpeRanks, &$cache)
    {
        if(array_key_exists($token, $cache)) return $cache[$token];
        $word = self::gptSplit($token);
        $init_len = count($word);
        $pairs = self::gptGetPairs($word);
        if(!$pairs) return $token;
        while (true){
            $minPairs = [];
            foreach($pairs as $pair){
                if(array_key_exists($pair[0] . ','. $pair[1], $bpeRanks)){
                    $rank = $bpeRanks[$pair[0] . ','. $pair[1]];
                    $minPairs[$rank] = $pair;
                }else{
                    $minPairs[10e10] = $pair;
                }
            }
            ksort($minPairs);
            $min_key = array_key_first($minPairs);
            foreach($minPairs as $mpi => $mp){
                if($mpi < $min_key) $min_key = $mpi;
            }
            $bigram = $minPairs[$min_key];
            if(!array_key_exists($bigram[0] . ',' . $bigram[1], $bpeRanks)) break;
            $first = $bigram[0];
            $second = $bigram[1];
            $new_word = [];
            $i = 0;
            while ($i < count($word)){
                $j = self::gptIndexOf($word, $first, $i);
                if ($j === -1){
                    $new_word = array_merge($new_word, array_slice($word, $i, null, true));
                    break;
                }
                if($i > $j){
                    $slicer = [];
                }elseif($j == 0){
                    $slicer = [];
                }else{
                    $slicer = array_slice($word, $i, $j - $i, true);
                }
                $new_word = array_merge($new_word, $slicer);
                if(count($new_word) > $init_len) break;
                $i = $j;
                if ($word[$i] === $first && $i < count($word) - 1 && $word[$i + 1] === $second){
                    $new_word[] = $first . $second;
                    $i = $i + 2;
                }else{
                    $new_word[] = $word[$i];
                    $i = $i + 1;
                }
            }
            if($word == $new_word) break;
            $word = $new_word;
            if (count($word) === 1){
                break;
            }else{
                $pairs = self::gptGetPairs($word);
            }
        }
        $word = implode(' ', $word);
        $cache[$token] = $word;
        return $word;
    }

    /**
     * @param array $wordList
     * @param $searchElement
     * @param int $fromIndex
     * @return int
     * gptIndexOf
     */
    private static function gptIndexOf(array $wordList, $searchElement, int $fromIndex):int
    {
        foreach($wordList as $index => $value){
            if($index < $fromIndex) continue;
            if($value == $searchElement) return $index;
        }
        return -1;
    }
}